package main

import (
	"database/sql"
	"fmt"
	"github.com/axgle/mahonia"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gocolly/colly/v2"
	"github.com/gocolly/colly/v2/extensions"
	"log"
	"math/rand"
	"net"
	"net/http"
	"time"
)

const BaseUrl = "http://fowap.goodweb.net.cn"

// 建立数据库连接
func dbSetupConnect() *sql.DB {
	db, err := sql.Open("mysql", "root:root@tcp(127.0.0.1:3306)/srx_xyxf?charset=utf8mb4&parseTime=True&loc=Local")
	if err != nil {
		fmt.Println(err.Error())
	}
	return db
}

const (
	PageCount = 8895
	Source    = "学佛网"
	Category  = "居士文章"
)

func main() {

	keyman(BaseUrl + "/page02b.asp")
	//page("http://fowap.goodweb.net.cn/news/news_more.asp?lm2=298", "李炳南")
	//list("http://fowap.goodweb.net.cn/news/news_more.asp?lm2=298", "李炳南", "第1页")
	//detail("http://fowap.goodweb.net.cn/news/news_view.asp?newsid=3782","李炳南","佛学问答类编 (净土第十二)")
}

func Convert(src string) string {
	srcCoder := mahonia.NewDecoder("gbk")
	srcResult := srcCoder.ConvertString(src)
	tagCoder := mahonia.NewDecoder("utf-8")
	_, cdata, _ := tagCoder.Translate([]byte(srcResult), true)
	return string(cdata)
}

func randInt(min int, max int) int {
	return min + rand.Intn(max-min)
}

func keyman(url string) {
	c := colly.NewCollector()
	extensions.RandomUserAgent(c)
	extensions.Referer(c)
	c.WithTransport(&http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   90 * time.Second,
			KeepAlive: 90 * time.Second,
		}).DialContext,
		MaxIdleConns:          1,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   90 * time.Second,
		ExpectContinueTimeout: 90 * time.Second,
	})

	// Before making a request print "Visiting ..."
	c.OnRequest(func(r *colly.Request) {
		fmt.Println("人物列表", r.URL.String())
	})

	c.OnError(func(e *colly.Response, err error) {
		log.Println("人物异常:", err)
		if e.StatusCode != http.StatusNotFound {
			keyman(url)
		}
	})

	c.OnHTML("div[class='block'] a[href]", func(e *colly.HTMLElement) {
		//html, _ := e.DOM.Html()
		//fmt.Println("html:  ", html)
		link := e.Attr("href")
		name := e.Text
		//fmt.Printf("link: %s  title: %s \n", link, title)
		if len(name) > 0 {
			page(link, name)
		}
	})

	// Start scraping
	_ = c.Visit(url)
}

func page(url string, name string) {
	c := colly.NewCollector()
	extensions.RandomUserAgent(c)
	extensions.Referer(c)
	c.WithTransport(&http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   90 * time.Second,
			KeepAlive: 90 * time.Second,
		}).DialContext,
		MaxIdleConns:          1,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   90 * time.Second,
		ExpectContinueTimeout: 90 * time.Second,
	})

	// Before making a request print "Visiting ..."
	c.OnRequest(func(r *colly.Request) {
		fmt.Println("页码", r.URL.String())
	})

	c.OnError(func(e *colly.Response, err error) {
		log.Println("页码异常:", err)
		if e.StatusCode != http.StatusNotFound {
			page(url, name)
		}
	})

	c.OnHTML("select option", func(e *colly.HTMLElement) {
		//html, _ := e.DOM.Html()
		//fmt.Println("html:  ", html)
		//查找A标签
		link := e.Attr("value")
		page := e.Text
		fmt.Printf("link: %s  page: %s \n", link, page)
		list(BaseUrl+link, name, page)
	})

	// Start scraping
	_ = c.Visit(url)
}

func list(url string, name string, page string) {
	c := colly.NewCollector()
	extensions.RandomUserAgent(c)
	extensions.Referer(c)
	c.WithTransport(&http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   90 * time.Second,
			KeepAlive: 90 * time.Second,
		}).DialContext,
		MaxIdleConns:          1,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   90 * time.Second,
		ExpectContinueTimeout: 90 * time.Second,
	})

	// Before making a request print "Visiting ..."
	c.OnRequest(func(r *colly.Request) {
		fmt.Println("访问列表", r.URL.String())
	})

	c.OnError(func(e *colly.Response, err error) {
		log.Println("列表异常:", err)
		if e.StatusCode != http.StatusNotFound {
			list(url, name, page)
		}
	})

	c.OnHTML("div[style='width=100%; text-align:left;'] a[href]", func(e *colly.HTMLElement) {
		//html, _ := e.DOM.Html()
		//fmt.Println("html:  ", html)
		//查找A标签
		link := e.Attr("href")
		title := e.Text
		//fmt.Printf("link: %s  title: %s \n", link, title)
		detail("http://fowap.goodweb.net.cn/news/"+link, name, title)
	})

	// Start scraping
	_ = c.Visit(url)
}

func detail(url string, name string, title string) {
	c := colly.NewCollector()
	extensions.RandomUserAgent(c)
	extensions.Referer(c)
	c.WithTransport(&http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   90 * time.Second,
			KeepAlive: 90 * time.Second,
		}).DialContext,
		MaxIdleConns:          1,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   90 * time.Second,
		ExpectContinueTimeout: 90 * time.Second,
	})

	c.OnRequest(func(r *colly.Request) {
		fmt.Println("访问详情", r.URL.String())
	})

	//插入数据库
	db := dbSetupConnect()
	defer db.Close()

	c.OnError(func(e *colly.Response, err error) {
		log.Println("详情异常:", err)
		if e.StatusCode != http.StatusNotFound {
			detail(url, name, title)
		}
	})

	c.OnHTML("div[class='style_txt']", func(e *colly.HTMLElement) {
		content := e.DOM.Text()
		//fmt.Println(content)
		//人物
		var keymanID int64
		kRow := db.QueryRow("SELECT id FROM keyman WHERE name = ?", name)
		if err := kRow.Scan(&keymanID); err != nil {
			res, err := db.Exec("INSERT INTO keyman(name) VALUES (?)", name)
			if err != nil {
				fmt.Println(err)
			}
			keymanID, _ = res.LastInsertId()
			fmt.Printf("========  %s : %d ========= 写入人物... \n", name, keymanID)
		}

		//分类
		var categoryID int64
		cRow := db.QueryRow("SELECT id FROM category WHERE name = ?", Category)
		if err := cRow.Scan(&categoryID); err != nil {
			res, err := db.Exec("INSERT INTO category(name) VALUES (?)", Category)
			if err != nil {
				fmt.Println(err)
			}
			categoryID, _ = res.LastInsertId()
			fmt.Printf("========  %s : %d ========= 写入分类... \n", Category, categoryID)
		}

		//文章
		_, err := db.Exec("REPLACE INTO article(title,content,keyman_id,keyman_name,category_id,category_name,source,hot,view,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)", title, content, keymanID, name, categoryID, Category, Source, randInt(1080, 3600), randInt(3600, 10080), time.Now(), time.Now())
		fmt.Printf("========  %s ========= 完成 \n", title)
		if err != nil {
			fmt.Println(err)
		}

	})

	_ = c.Visit(url)
}
